Introduction¶
This notebook facilitates the manual curation of sample alignment.
In [1]:
import deltascope.alignment as ut
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import h5py
import os
import re
import time
import tqdm
Setup¶
Parameters¶
In [2]:
# --------------------------------
# -------- User input ------------
# --------------------------------
param = {
'gthresh':0.5,
'scale':[1,1,1],
'microns':[0.16,0.16,0.21],
'mthresh':0.5,
'radius':10,
'comp_order':[0,2,1],
'fit_dim':['x','z'],
'deg':2,
# Don't forget to modify this with a specific sample name
'expname':'yot'
}
Directories¶
In [3]:
# --------------------------------
# -------- User input ------------
# --------------------------------
# Specify file paths to directories containing probability files
# after processing by ilastik
gfap = '.\data\ZRF1 You-Too HZ H5 files\Prob_7-29'
at = os.path.abspath('.\data\AT You-Too HZ Mutant H5 files\Prob_7-26')
# Specify root directory where output will be saved
root = '.\data'
root
Out[3]:
In [4]:
# Output directory with timestamp
outname = 'Output_'+param['expname']+time.strftime("%m-%d-%H-%M",
time.localtime())
# Create output directory
outdir = os.path.join(root,outname)
os.mkdir(outdir)
Extract list of files¶
In [5]:
Dat = {}
for f in os.listdir(at):
if 'h5' in f:
num = re.findall(r'\d+',f.split('.')[0])[-1]
Dat[num] = os.path.join(at,f)
Dat
Out[5]:
In [6]:
Dzrf = {}
for f in os.listdir(gfap):
if 'h5' in f:
num = re.findall(r'\d+',f.split('.')[0])[-1]
Dzrf[num] = os.path.join(gfap,f)
In [7]:
# Extract list of filename keys
klist = list(Dat.keys())
In [8]:
# Create dictionaries to contain the deltascope brain object for each sample
Dbat = {}
Dbzrf = {}
PCA: Import raw data and perform preprocessing¶
You-too processing fo samples 19 21, and 172, with radius of 5, vs 10, to deal with the loss of all data in the array caused by radius 10 median filter causing the small sparse axon crossing to be comletely lost to the filter.
In [9]:
%%time
for k in tqdm.tqdm(klist):
if k not in list(Dbat.keys()):
if k in ['19','21','172']:
param['radius'] = 5
else:
param['radius'] = 10
try:
Dbat[k] = ut.preprocess(Dat[k],param)
Dbzrf[k] = ut.preprocess(Dzrf[k],param,pca=Dbat[k].pcamed,
mm=Dbat[k].mm,vertex=Dbat[k].vertex)
except:
print(k,'caused an error')
else:
print(k,'already processed')
Define experiment specific functions¶
In [11]:
''' Define wrapper functions for starting and saving to minimize the number
of inputs that the user needs to type for each call of the function.'''
def start(k):
return(ut.start(k,Dbat,[Dbzrf],im=True))
def save_both(k,dfa,dfb):
ut.save_both(k,dfa,dfb,outdir,param['expname'])
'''Save model parameters for each file to a dataframe that can be
exported for later reference.'''
model = pd.DataFrame({'a':[],'b':[],'c':[]})
def save_model(k,mm,model):
row = pd.Series({'a':mm[0],'b':mm[1],'c':mm[2]},name=k)
model = model.append(row)
return(model)
'''Define a function that can both fit a model and plot it on an existing plot'''
def fit_model(axi,df,mm=None):
if mm == None:
mm = np.polyfit(df.x,df.z,2)
p = np.poly1d(mm)
xrange = np.arange(np.min(df.x),np.max(df.x))
axi.plot(xrange,p(xrange),c='m')
return(mm)
'''Take a set of points and transform to a dataframe format for ease of access.'''
def pick_pts(x1,z1,vx,vz,x2,z2):
pts = pd.DataFrame({'x':[x1,vx,x2],'z':[z1,vz,z2]})
return(pts)
You-too processing fo samples 19 21, and 172, with radius of 5, vs 20, to deal with the loss of all data in the array caused by radius 20 median filter causing the small sparse axon crossing to be comletely lost to the filter.
In [12]:
k,df,Ldf,ax = start('19')
In [13]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [14]:
pts.iloc[0].x=40
In [15]:
pts.iloc[0].z=15
In [16]:
df1,Ldf1,ax = ut.revise_pts(df,Ldf,'z',pts=pts)
In [17]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [18]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [19]:
k,df,Ldf,ax = start('21')
In [20]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [21]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [22]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [23]:
k,df,Ldf,ax = start('172')
In [24]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [25]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
Processing of remaining you-too samples with 10 median filter
In [26]:
klist
Out[26]:
In [27]:
k,df,Ldf,ax = start('01')
In [28]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [29]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [30]:
klist
Out[30]:
In [31]:
k,df,Ldf,ax = start('02')
In [32]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [33]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [34]:
k,df,Ldf,ax = start('03')
In [35]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [36]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [37]:
k,df,Ldf,ax = start('04')
In [38]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf)
In [39]:
pts = pick_pts(-60,8,-18,-10,25,8)
In [40]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [41]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [42]:
klist
Out[42]:
In [43]:
k,df,Ldf,ax = start('06')
In [44]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf)
In [45]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [46]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [47]:
klist
Out[47]:
In [48]:
k,df,Ldf,ax = start('112')
In [49]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf)
In [50]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [51]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [52]:
k,df,Ldf,ax = start('24')
In [53]:
df2,Ldf2,ax,p = ut.check_yz(df1,Ldf1)
In [54]:
df1,Ldf1,pts,ax = ut.check_pts(df,Ldf,'z')
In [55]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2)
In [56]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [57]:
klist
Out[57]:
In [58]:
k,df,Ldf,ax = start('152')
In [59]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf)
In [60]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [61]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [62]:
klist
Out[62]:
In [63]:
k,df,Ldf,ax = start('317')
In [64]:
df1,Ldf1,ax,p = ut.check_yz(df,Ldf)
In [65]:
p1=1,0
In [66]:
p1
Out[66]:
In [67]:
df1,Ldf1,ax,p1= ut.check_yz(df,Ldf, mm=p1)
In [68]:
pts.iloc[1].x=30
In [69]:
pts.iloc[1].z=9
In [70]:
df2,Ldf2,ax = ut.revise_pts(df1,Ldf1,'z',pts=pts)
In [71]:
pts = pick_pts(-45,9,-8,-5,30,9)
In [72]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2,pts=pts)
In [73]:
p1=-.3,0
In [74]:
df1,Ldf1,ax,p1 = ut.check_yz(df,Ldf,mm=p1)
In [75]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [76]:
klist
Out[76]:
In [77]:
k,df,Ldf,ax = start('318')
In [78]:
p1=.3,0
In [79]:
df1,Ldf1,ax,p1 = ut.check_yz(df,Ldf,mm=p1)
In [80]:
df2,Ldf2,pts,ax = ut.check_pts(df1,Ldf1,'z')
In [81]:
pts.iloc[1].x = -40
In [82]:
pts.iloc[1].z=30
In [83]:
pts.iloc[0].x
Out[83]:
In [84]:
pts.iloc[0].z=25
In [85]:
df2,Ldf2,ax = ut.revise_pts(df1,Ldf1,'z',pts=pts)
In [86]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2)
In [87]:
pts = pick_pts(-42,30,-5,-5,41,30)
In [88]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2,pts=pts)
In [89]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [90]:
klist
Out[90]:
In [91]:
k,df,Ldf,ax = start('319')
In [92]:
p1=-.4,-5
In [93]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [94]:
pts = pick_pts(-35,22,0,-2,35,22)
In [95]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [96]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [97]:
klist
Out[97]:
In [98]:
k,df,Ldf,ax = start('320')
In [99]:
p1=-.5,-10
In [100]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [101]:
pts = pick_pts(-38,28,0,-10,45,28)
In [102]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [103]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [104]:
klist
Out[104]:
In [105]:
k,df,Ldf,ax= start('321')
In [106]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [107]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [108]:
k,df,Ldf,ax = start('323')
In [109]:
df1,Ldf1,mm,ax = ut.ch_vertex(df,Ldf)
In [110]:
model = save_model(k,mm,model)
save_both(k,df1,Ldf1[0])
In [111]:
k,df,Ldf,ax = start('324')
In [112]:
p1=.4,10
In [113]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [114]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [115]:
pts = pick_pts(-33,18,0,-4,33,18)
In [116]:
df3,Ldf3,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [117]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [118]:
k,df,Ldf,ax = start('325')
In [119]:
p1=.5,10
In [120]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [121]:
pts = pick_pts(-30,12,0,-8,34,12)
In [122]:
df3,Ldf3,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [123]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [124]:
klist
Out[124]:
In [125]:
k,df,Ldf,ax = start('326')
In [126]:
pts = pick_pts(-35,20,-7,-7,38,20)
In [127]:
df3,Ldf3,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [128]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [129]:
k,df,Ldf,ax = start('327')
In [130]:
pts = pick_pts(-37,35,-2,-5,34,33)
In [131]:
df3,Ldf3,mm,ax = ut.ch_vertex(df,Ldf,pts=pts)
In [132]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [133]:
k,df,Ldf,ax= start('328')
In [134]:
pts = pick_pts(-37,25,-8,-6,36,25)
In [135]:
df3,Ldf3,mm,ax = ut.ch_vertex(df,Ldf,pts=pts)
In [136]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [137]:
k,df,Ldf,ax = start('329')
In [138]:
p1=.4,0
In [139]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [140]:
pts = pick_pts(-43,40,-4,-6,41,40)
In [141]:
df3,Ldf3,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [142]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [143]:
k,df,Ldf,ax = start('330')
In [144]:
df2,Ldf2,mm,ax = ut.ch_vertex(df,Ldf)
In [145]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [146]:
k,df,Ldf,ax = start('331')
In [147]:
pts.iloc[0].z=33
In [148]:
df2,Ldf2,ax = ut.revise_pts(df,Ldf,'z',pts=pts)
In [149]:
pts = pick_pts(-45,30,-12,-10,38,30)
In [150]:
df3,Ldf3,mm,ax = ut.ch_vertex(df2,Ldf2,pts=pts)
In [151]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [152]:
k,df,Ldf,ax = start('332')
In [153]:
p1=1.9,5
In [154]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [155]:
pts = pick_pts(-35,20,0,-9,40,20)
In [156]:
df3,Ldf3,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [157]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [158]:
k,df,Ldf,ax= start('333')
In [159]:
df2,Ldf2,mm,ax = ut.ch_vertex(df,Ldf)
In [160]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [161]:
k,df,Ldf,ax = start('334')
In [162]:
p1=1.5,0
In [163]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [164]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [165]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [166]:
k,df,Ldf,ax = start('335')
In [167]:
p1=1.1,0
In [168]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [169]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [170]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [171]:
k,df,Ldf,ax = start('336')
In [172]:
pts = pick_pts(-47,30,0,-10,47,30)
In [173]:
df3,Ldf3,mm,ax = ut.ch_vertex(df,Ldf,pts=pts)
In [174]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [175]:
k,df,Ldf,ax= start('337')
In [ ]:
In [176]:
p1=2,0
In [177]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [178]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [179]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [180]:
k,df,Ldf,ax = start('338')
In [181]:
p1=1.3,0
In [182]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [183]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [184]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [185]:
k,df,Ldf,ax = start('339')
In [186]:
p1=2.5,0
In [187]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [188]:
df2,Ldf2,mm,ax = ut.ch_vertex(df1,Ldf1)
In [189]:
model = save_model(k,mm,model)
save_both(k,df2,Ldf2[0])
In [190]:
k,df,Ldf,ax = start('340')
In [191]:
p1=-.6,0
In [192]:
df1,Ldf1,ax,p1 =ut.check_yz(df,Ldf,mm=p1)
In [193]:
pts = pick_pts(-42,25,0,-7,45,25)
In [194]:
df3,Ldf3,mm,ax = ut.ch_vertex(df1,Ldf1,pts=pts)
In [195]:
model = save_model(k,mm,model)
save_both(k,df3,Ldf3[0])
In [196]:
model
Out[196]:
In [198]:
model.to_csv(os.path.join(outdir,'model.csv'))